#Scopus Database Query By Journal Article
library(XML)
library(proto)
library(magrittr)
library(broom)
library(plyr)
library(dplyr)
library(ggplot2)
library(reshape2)
library(httr)
library(stringr)
library(chron)
library(vegan)
library(knitr)
library(bipartite)
library(sna)
library(igraph)
library(knitr)
library(gridExtra)
library(GGally)
library(stringr)
library(networkD3)
#source functions
source("Funtions.R")

#set knitr options
opts_chunk$set(echo=T,cache=F,fig.align='center',fig.height=15,fig.width=14,warning=F,message=F)

Read in data. Processed from ByJournal.R

#journal class
journaldf<-read.csv("C:/Users/Ben/Dropbox/FacultyNetwork/JournalID.csv",row.names=1)

tocompare<-read.table("C:/Users/Ben/Dropbox/FacultyNetwork/ParsedDataID.csv",row.names=NULL,header=T,sep=",",fill=T)

#what does it look like:
tail(tocompare)
##                   Journal                   DOI Order      Author
## 4763443 Zoologica Scripta SCOPUS_ID:84927938292     3  7005747774
## 4763444 Zoologica Scripta SCOPUS_ID:84927938292     4  8692718700
## 4763445 Zoologica Scripta SCOPUS_ID:84927938292     2 56596457600
## 4763446 Zoologica Scripta SCOPUS_ID:84927938292     1 54890209300
## 4763447 Zoologica Scripta SCOPUS_ID:84927938292     6  6701851552
## 4763448 Zoologica Scripta SCOPUS_ID:84927938292     5  6506098243
##                                                                    Affiliation
## 4763443                                      Universita degli Studi del Molise
## 4763444                                                     Universita di Pisa
## 4763445                             Museo Civico di Storia Naturale di Trieste
## 4763446 Dipartimento di Biologia Univerisità degli Studi di Napoli Federico II
## 4763447                           Universita degli Studi di Napoli Federico II
## 4763448                                                 University of Belgrade
##         Citations Year   Class h5.index h5.median
## 4763443         0 2015 Zoology       23        27
## 4763444         0 2015 Zoology       23        27
## 4763445         0 2015 Zoology       23        27
## 4763446         0 2015 Zoology       23        27
## 4763447         0 2015 Zoology       23        27
## 4763448         0 2015 Zoology       23        27
dim(tocompare)
## [1] 4763448      10
#remove biogeneral i think for now
tocompare<-tocompare[!tocompare$Class %in% "Biogeneral",]

Basic data cleaning. We only want records from active authors. Atleast 5 publications in the entire record.

#filter authors
#distribution of publication
keep<-names(which(table(tocompare$Author)>3))
tocompare<-droplevels(tocompare[tocompare$Author %in% keep,])

#remove duplicates.
tocompare<-tocompare[!duplicated(tocompare),]

#in case the column names in there 
tocompare<-droplevels(tocompare[!tocompare$Journal %in% "DOI",])
dim(tocompare)
## [1] 2715847      10
j_class<-read.csv("Class.csv",row.names=1)

#in case there are malformed classes
tocompare<-droplevels(tocompare[tocompare$Class %in% j_class$Class,])

#take out malformed lines
tocompare<-tocompare[which(!str_detect(tocompare$Journal, "SCOPUS")),]

#take year 2015
tocompare<-tocompare[!tocompare$Year %in% 2015,]

tocompare<-droplevels(tocompare[which(!str_detect(tocompare$h5.index, "SCOPUS")),])
dim(tocompare)
## [1] 2659518      10

Basic descriptive stats on results

#How many journals
paste("Number of Journals:",length(unique(tocompare$Journal)))
## [1] "Number of Journals: 578"
#How many authors
paste("Number of Authors:",length(unique(tocompare$Author)))
## [1] "Number of Authors: 327175"
#How many papers
paste("Number of Papers:",length(unique(tocompare$DOI)))
## [1] "Number of Papers: 1325937"
ta<-sort(table(tocompare$Journal))
print("Most published journals")
## [1] "Most published journals"
tail(ta)
## 
##             Journal Of Virology                        Oncogene 
##                           36572                           40980 
##         Atmospheric Environment          Nucleic Acids Research 
##                           45494                           54840 
##  Molecular And Cellular Biology Journal Of Biological Chemistry 
##                           66726                           77452

How many papers from each discipline over time?

class_year<-group_by(tocompare,Class,Year) %>% summarize(Papers=length(unique(DOI)))
ggplot(class_year,aes(x=as.factor(Year),y=Papers,col=Class,group=Class)) + geom_line() + theme_bw() + facet_wrap(~Class,scale="free_y",ncol=4) + labs(x="Year")

plot of chunk unnamed-chunk-6

ggsave("Figures/Papers_Year.svg",dpi=300)

Create matrix of authors in each class - analagous to the site by species matrix used in ecology

siteXspp<-as.data.frame.array(table(tocompare$Author,tocompare$Class))
dim(siteXspp)
## [1] 327175     38

Dissimalarity among classes

Use the abundance of papers by each author to calculate niche overlap (dist=1-Horn’s) between classes.

Low overlap=0 High overlap=1

#Compare disciplines
topics<-1-as.matrix(vegdist(t(siteXspp),"horn"))

Visualize interactions.

g<-graph.adjacency(topics,"undirected",weighted=TRUE)

g<-simplify(g)

# set labels and degrees of vertices
V(g)$label <- V(g)$name
V(g)$degree <- degree(g)

V(g)$label.color <- rgb(0, 0, .2, .8)
V(g)$frame.color <- NA
egam=E(g)$weight/max(E(g)$weight)
E(g)$color<-rgb(0,1,0,alpha=E(g)$weight/max(E(g)$weight),maxColorValue=1)

ramp <- colorRamp(c("blue","red"),alpha=T)

E(g)$color = apply(ramp(E(g)$weight), 1, function(x) rgb(x[1]/255,x[2]/255,x[3]/255,alpha=T) )


#If you need to delete
g.copy <- delete.edges(g, which(E(g)$weight<.05))
#width
width<-(E(g.copy)$weight/max(E(g.copy)$weight))*8

#label sizes
V(g.copy)$degree <- degree(g.copy)
V(g.copy)$label.cex <- V(g.copy)$degree / max(V(g.copy)$degree)*.5+.5

#get vertex size
size<-group_by(tocompare,Class) %>% summarize(Papers=length(unique(DOI)))
size<-round(as.numeric(table(tocompare$Class))/max(table(tocompare$Class))*5)+2

#scale
plot.igraph(g.copy,vertex.size=size,edge.width=width)

plot of chunk unnamed-chunk-9

#try a couple different layouts
# plot the graph in layout1
layout1 <- layout.fruchterman.reingold(g.copy,niter=10000,area=vcount(g.copy)^2.3)

plot(g.copy,edge.width=width,vertex.size=size,layout=layout1,vertex.color="grey40")

plot of chunk unnamed-chunk-9

# plot the graph in layout1
layout2 <- layout.reingold.tilford(g.copy,circular=T)

plot(g.copy,edge.width=width,vertex.size=size,layout=layout2,vertex.color="grey70")

plot of chunk unnamed-chunk-9

# plot the graph in layout3
layout3 <- layout.kamada.kawai(g.copy,niter=5000)

plot(g.copy,edge.width=width,vertex.size=size,layout=layout3,vertex.color="grey40")

plot of chunk unnamed-chunk-9

#save full  network
svg(filename = "Figures/Overall_NetworkIgraph.svg",width=10,height=10)
plot(g.copy,edge.width=width,vertex.size=size,layout=layout1,vertex.color="grey40")
dev.off()
## pdf 
##   2
jpeg(filename = "Figures/Overall_NetworkIgraph.jpeg",res=300,height=12,width=10,units="in")
plot(g.copy,edge.width=width,vertex.size=size,layout=layout1,vertex.color="grey40")
dev.off()
## pdf 
##   2

View as a dendrogram

wt <- walktrap.community(g, modularity=TRUE)
dend <- as.dendrogram(wt, use.modularity=TRUE)
plot(as.hclust(dend))

plot of chunk unnamed-chunk-10

Calculate network statistics

We are interested in the centrality, modularity and compartamentalization of the biological sciences

between_class<-betweenness(g.copy)
degree_class<-degree(g.copy)
closeness_class<-closeness(g.copy)
eigenV<-evcent(g.copy)$vector
vdat<-data.frame(Class=names(between_class),Between=between_class,Degree=degree_class,Closeness=closeness_class,Eigen=eigenV)

Correlation among importance measures.

ggpairs(vdat[,-1])

plot of chunk unnamed-chunk-12

#reoroder levels
vdat$Class<-factor(vdat$Class,levels=vdat[order(vdat$Between,vdat$Degree),"Class"])

Top actors for each statistic

mdat<-melt(vdat)
colnames(mdat)<-c("Class","Metric","Score")
#order and plot
dt<-group_by(mdat,Metric) %>% mutate(Svalue=as.numeric(scale(Score))) %>% group_by(Metric,Class) %>% arrange(Score)

ggplot(dt,aes(y=Class,x=Svalue)) + geom_bar_horz(stat="identity",position="identity") + facet_grid(.~Metric) + labs(x="Z-score")

plot of chunk unnamed-chunk-13

ggsave("Figures/OveralMetrics.jpg",dpi=300,height=10,width=12)
ggsave("Figures/OveralMetrics.svg",dpi=300)

Network stats over time

#split into a time frame 5 years?

m<-seq(1995,2014,2)

tocompare$Time<-cut(as.numeric(as.character(tocompare$Year)),breaks=m,labels=m[1:length(m)-1])

yearcompare<-split(tocompare,tocompare$Time)

#all over 

#caculate degree distribution
dd<-melt(sapply(yearcompare,CalcDD))
ggplot(dd,aes(x=Var1,y=value,col=as.factor(L1))) + geom_line(size=.5) + geom_point(size=4,aes(group=as.factor(L1))) + theme_bw() + xlab("Node Degree") + ggtitle("Degree Distribution") + labs(col="Year") 

plot of chunk unnamed-chunk-15

yearstats<-lapply(yearcompare,calcN)

plot of chunk unnamed-chunk-15plot of chunk unnamed-chunk-15plot of chunk unnamed-chunk-15plot of chunk unnamed-chunk-15plot of chunk unnamed-chunk-15plot of chunk unnamed-chunk-15plot of chunk unnamed-chunk-15plot of chunk unnamed-chunk-15plot of chunk unnamed-chunk-15

head(yearstats)
## $`1995`
##                                                                         Class
## Agronomy Crop Science                                   Agronomy Crop Science
## Animal Behavior                                               Animal Behavior
## Animal Husbandry                                             Animal Husbandry
## Atmospheric Sciences                                     Atmospheric Sciences
## Biochemistry                                                     Biochemistry
## Biodiversity & Conservation Biology       Biodiversity & Conservation Biology
## Bioinformatics                                                 Bioinformatics
## Biophysics                                                         Biophysics
## Biotechnology                                                   Biotechnology
## Birds                                                                   Birds
## Botany                                                                 Botany
## Cell Biology                                                     Cell Biology
## Developmental Biology & Embryology         Developmental Biology & Embryology
## Ecology                                                               Ecology
## Environmental & Geological Engineering Environmental & Geological Engineering
## Environmental Sciences                                 Environmental Sciences
## Evolutionary Biology                                     Evolutionary Biology
## Food Science & Technology                           Food Science & Technology
## Forests & Forestry                                         Forests & Forestry
## Geochemistry & Mineralogy                           Geochemistry & Mineralogy
## Geology                                                               Geology
## Hydrology                                                           Hydrology
## Insects & Arthropods                                     Insects & Arthropods
## Marine Sciences Fisheries                           Marine Sciences Fisheries
## Microbiology                                                     Microbiology
## Molecular Biology                                           Molecular Biology
## Mycology                                                             Mycology
## Oceanography                                                     Oceanography
## Paleontology                                                     Paleontology
## Pest Control & Pesticides                           Pest Control & Pesticides
## Plant Pathology                                               Plant Pathology
## Proteomics & Peptides                                   Proteomics & Peptides
## Soil Sciences                                                   Soil Sciences
## Sustainable Development                               Sustainable Development
## Sustainable Energy                                         Sustainable Energy
## Virology                                                             Virology
## Wood Science & Technology                           Wood Science & Technology
## Zoology                                                               Zoology
##                                        Between Degree Closeness     Eigen
## Agronomy Crop Science                       24      3 0.0021109 0.000e+00
## Animal Behavior                             35      4 0.0021493 7.636e-04
## Animal Husbandry                            27      2 0.0021458 7.274e-04
## Atmospheric Sciences                         0      2 0.0021367 0.000e+00
## Biochemistry                                 1      2 0.0007515 7.035e-01
## Biodiversity & Conservation Biology          0      3 0.0021423 0.000e+00
## Bioinformatics                               0      0 0.0007112 2.979e-04
## Biophysics                                   0      1 0.0007310 7.026e-04
## Biotechnology                               49      3 0.0021461 1.383e-03
## Birds                                        0      3 0.0021493 3.369e-05
## Botany                                      46      2 0.0021187 6.297e-04
## Cell Biology                                 0      2 0.0007513 9.387e-01
## Developmental Biology & Embryology           0      0 0.0007112 2.979e-04
## Ecology                                    177      7 0.0021548 0.000e+00
## Environmental & Geological Engineering       0      1 0.0021445 3.073e-04
## Environmental Sciences                     162      6 0.0021520 0.000e+00
## Evolutionary Biology                         0      2 0.0021449 0.000e+00
## Food Science & Technology                   25      2 0.0021450 1.078e-03
## Forests & Forestry                           0      0 0.0007112 2.979e-04
## Geochemistry & Mineralogy                    0      1 0.0021151 7.143e-05
## Geology                                     47      3 0.0021347 4.324e-04
## Hydrology                                    0      0 0.0007112 2.979e-04
## Insects & Arthropods                       100      2 0.0021511 2.441e-04
## Marine Sciences Fisheries                  118      3 0.0021532 0.000e+00
## Microbiology                                22      2 0.0021442 9.731e-04
## Molecular Biology                            0      2 0.0007513 1.000e+00
## Mycology                                     0      0 0.0007112 2.979e-04
## Oceanography                                87      4 0.0021468 0.000e+00
## Paleontology                                 0      1 0.0021125 0.000e+00
## Pest Control & Pesticides                   84      2 0.0021348 6.581e-05
## Plant Pathology                             66      3 0.0021254 0.000e+00
## Proteomics & Peptides                        0      1 0.0007310 7.026e-04
## Soil Sciences                                0      1 0.0020801 0.000e+00
## Sustainable Development                      0      1 0.0021461 4.057e-04
## Sustainable Energy                           0      1 0.0021395 5.235e-04
## Virology                                     0      0 0.0007112 2.979e-04
## Wood Science & Technology                    0      0 0.0007112 2.979e-04
## Zoology                                     23      4 0.0021480 8.343e-05
## 
## $`1997`
##                                                                         Class
## Agronomy Crop Science                                   Agronomy Crop Science
## Animal Behavior                                               Animal Behavior
## Animal Husbandry                                             Animal Husbandry
## Atmospheric Sciences                                     Atmospheric Sciences
## Biochemistry                                                     Biochemistry
## Biodiversity & Conservation Biology       Biodiversity & Conservation Biology
## Bioinformatics                                                 Bioinformatics
## Biophysics                                                         Biophysics
## Biotechnology                                                   Biotechnology
## Birds                                                                   Birds
## Botany                                                                 Botany
## Cell Biology                                                     Cell Biology
## Developmental Biology & Embryology         Developmental Biology & Embryology
## Ecology                                                               Ecology
## Environmental & Geological Engineering Environmental & Geological Engineering
## Environmental Sciences                                 Environmental Sciences
## Evolutionary Biology                                     Evolutionary Biology
## Food Science & Technology                           Food Science & Technology
## Forests & Forestry                                         Forests & Forestry
## Geochemistry & Mineralogy                           Geochemistry & Mineralogy
## Geology                                                               Geology
## Hydrology                                                           Hydrology
## Insects & Arthropods                                     Insects & Arthropods
## Marine Sciences Fisheries                           Marine Sciences Fisheries
## Microbiology                                                     Microbiology
## Molecular Biology                                           Molecular Biology
## Mycology                                                             Mycology
## Oceanography                                                     Oceanography
## Paleontology                                                     Paleontology
## Pest Control & Pesticides                           Pest Control & Pesticides
## Plant Pathology                                               Plant Pathology
## Proteomics & Peptides                                   Proteomics & Peptides
## Soil Sciences                                                   Soil Sciences
## Sustainable Development                               Sustainable Development
## Sustainable Energy                                         Sustainable Energy
## Virology                                                             Virology
## Wood Science & Technology                           Wood Science & Technology
## Zoology                                                               Zoology
##                                        Between Degree Closeness     Eigen
## Agronomy Crop Science                       23      3 0.0019900 8.694e-04
## Animal Behavior                             41      5 0.0019942 1.525e-04
## Animal Husbandry                            27      2 0.0019904 5.283e-04
## Atmospheric Sciences                         0      2 0.0019789 0.000e+00
## Biochemistry                                 0      2 0.0007515 7.310e-01
## Biodiversity & Conservation Biology         24      4 0.0019887 0.000e+00
## Bioinformatics                               0      0 0.0007112 8.761e-05
## Biophysics                                   0      1 0.0007309 1.770e-04
## Biotechnology                               23      2 0.0019896 2.975e-04
## Birds                                       32      3 0.0019942 0.000e+00
## Botany                                     101      3 0.0019957 1.581e-04
## Cell Biology                                 0      2 0.0007513 9.432e-01
## Developmental Biology & Embryology           0      0 0.0007112 8.761e-05
## Ecology                                    162      7 0.0019984 0.000e+00
## Environmental & Geological Engineering       0      1 0.0019887 3.196e-05
## Environmental Sciences                     128      5 0.0019942 3.824e-04
## Evolutionary Biology                         0      3 0.0019910 0.000e+00
## Food Science & Technology                   23      2 0.0019885 5.556e-04
## Forests & Forestry                           0      1 0.0019837 0.000e+00
## Geochemistry & Mineralogy                    0      1 0.0019596 1.386e-04
## Geology                                     45      3 0.0019783 2.807e-04
## Hydrology                                    0      0 0.0007112 8.761e-05
## Insects & Arthropods                         0      1 0.0019544 4.412e-04
## Marine Sciences Fisheries                  103      3 0.0019966 0.000e+00
## Microbiology                                18      2 0.0019873 4.969e-04
## Molecular Biology                            0      2 0.0007512 1.000e+00
## Mycology                                     0      0 0.0007112 8.761e-05
## Oceanography                                63      4 0.0019891 0.000e+00
## Paleontology                                 0      1 0.0019635 1.205e-04
## Pest Control & Pesticides                   23      2 0.0019787 8.262e-04
## Plant Pathology                             44      3 0.0019885 6.083e-04
## Proteomics & Peptides                        0      1 0.0007309 1.770e-04
## Soil Sciences                                0      1 0.0019630 2.063e-04
## Sustainable Development                      0      0 0.0007112 8.761e-05
## Sustainable Energy                           0      0 0.0007112 8.761e-05
## Virology                                     0      0 0.0007112 8.761e-05
## Wood Science & Technology                    0      0 0.0007112 8.761e-05
## Zoology                                     10      4 0.0019923 0.000e+00
## 
## $`1999`
##                                                                         Class
## Agronomy Crop Science                                   Agronomy Crop Science
## Animal Behavior                                               Animal Behavior
## Animal Husbandry                                             Animal Husbandry
## Atmospheric Sciences                                     Atmospheric Sciences
## Biochemistry                                                     Biochemistry
## Biodiversity & Conservation Biology       Biodiversity & Conservation Biology
## Bioinformatics                                                 Bioinformatics
## Biophysics                                                         Biophysics
## Biotechnology                                                   Biotechnology
## Birds                                                                   Birds
## Botany                                                                 Botany
## Cell Biology                                                     Cell Biology
## Developmental Biology & Embryology         Developmental Biology & Embryology
## Ecology                                                               Ecology
## Environmental & Geological Engineering Environmental & Geological Engineering
## Environmental Sciences                                 Environmental Sciences
## Evolutionary Biology                                     Evolutionary Biology
## Food Science & Technology                           Food Science & Technology
## Forests & Forestry                                         Forests & Forestry
## Geochemistry & Mineralogy                           Geochemistry & Mineralogy
## Geology                                                               Geology
## Hydrology                                                           Hydrology
## Insects & Arthropods                                     Insects & Arthropods
## Marine Sciences Fisheries                           Marine Sciences Fisheries
## Microbiology                                                     Microbiology
## Molecular Biology                                           Molecular Biology
## Mycology                                                             Mycology
## Oceanography                                                     Oceanography
## Paleontology                                                     Paleontology
## Pest Control & Pesticides                           Pest Control & Pesticides
## Plant Pathology                                               Plant Pathology
## Proteomics & Peptides                                   Proteomics & Peptides
## Soil Sciences                                                   Soil Sciences
## Sustainable Development                               Sustainable Development
## Sustainable Energy                                         Sustainable Energy
## Virology                                                             Virology
## Wood Science & Technology                           Wood Science & Technology
## Zoology                                                               Zoology
##                                        Between Degree Closeness     Eigen
## Agronomy Crop Science                        4      3 0.0008215 1.387e-03
## Animal Behavior                             33      5 0.0014501 0.000e+00
## Animal Husbandry                            18      2 0.0014485 1.662e-05
## Atmospheric Sciences                        18      3 0.0014479 0.000e+00
## Biochemistry                                 1      2 0.0007515 6.836e-01
## Biodiversity & Conservation Biology          2      4 0.0014482 0.000e+00
## Bioinformatics                               0      0 0.0007112 0.000e+00
## Biophysics                                   0      1 0.0007309 0.000e+00
## Biotechnology                               25      2 0.0014497 0.000e+00
## Birds                                        4      2 0.0014475 0.000e+00
## Botany                                       6      2 0.0008215 1.428e-04
## Cell Biology                                 0      2 0.0007514 9.452e-01
## Developmental Biology & Embryology           0      0 0.0007112 0.000e+00
## Ecology                                     67      6 0.0014510 2.284e-04
## Environmental & Geological Engineering       0      1 0.0014498 0.000e+00
## Environmental Sciences                      96      5 0.0014518 8.535e-04
## Evolutionary Biology                         0      3 0.0014488 0.000e+00
## Food Science & Technology                   20      2 0.0014490 8.402e-05
## Forests & Forestry                          12      2 0.0014495 0.000e+00
## Geochemistry & Mineralogy                    0      1 0.0014393 1.018e-05
## Geology                                     35      3 0.0014467 3.505e-04
## Hydrology                                    0      1 0.0014453 9.666e-05
## Insects & Arthropods                         0      1 0.0008206 4.694e-04
## Marine Sciences Fisheries                   60      3 0.0014512 0.000e+00
## Microbiology                                21      2 0.0014492 0.000e+00
## Molecular Biology                            0      2 0.0007513 1.000e+00
## Mycology                                     0      0 0.0007112 0.000e+00
## Oceanography                                76      4 0.0014507 7.076e-04
## Paleontology                                 0      1 0.0014382 4.566e-05
## Pest Control & Pesticides                    4      2 0.0008213 7.964e-04
## Plant Pathology                              6      3 0.0008215 6.410e-04
## Proteomics & Peptides                        0      1 0.0007309 0.000e+00
## Soil Sciences                                0      1 0.0008207 7.466e-04
## Sustainable Development                      0      0 0.0007112 0.000e+00
## Sustainable Energy                           0      0 0.0007112 0.000e+00
## Virology                                     0      0 0.0007112 0.000e+00
## Wood Science & Technology                    0      0 0.0007112 0.000e+00
## Zoology                                      0      4 0.0014488 0.000e+00
## 
## $`2001`
##                                                                         Class
## Agronomy Crop Science                                   Agronomy Crop Science
## Animal Behavior                                               Animal Behavior
## Animal Husbandry                                             Animal Husbandry
## Atmospheric Sciences                                     Atmospheric Sciences
## Biochemistry                                                     Biochemistry
## Biodiversity & Conservation Biology       Biodiversity & Conservation Biology
## Bioinformatics                                                 Bioinformatics
## Biophysics                                                         Biophysics
## Biotechnology                                                   Biotechnology
## Birds                                                                   Birds
## Botany                                                                 Botany
## Cell Biology                                                     Cell Biology
## Developmental Biology & Embryology         Developmental Biology & Embryology
## Ecology                                                               Ecology
## Environmental & Geological Engineering Environmental & Geological Engineering
## Environmental Sciences                                 Environmental Sciences
## Evolutionary Biology                                     Evolutionary Biology
## Food Science & Technology                           Food Science & Technology
## Forests & Forestry                                         Forests & Forestry
## Geochemistry & Mineralogy                           Geochemistry & Mineralogy
## Geology                                                               Geology
## Hydrology                                                           Hydrology
## Insects & Arthropods                                     Insects & Arthropods
## Marine Sciences Fisheries                           Marine Sciences Fisheries
## Microbiology                                                     Microbiology
## Molecular Biology                                           Molecular Biology
## Mycology                                                             Mycology
## Oceanography                                                     Oceanography
## Paleontology                                                     Paleontology
## Pest Control & Pesticides                           Pest Control & Pesticides
## Plant Pathology                                               Plant Pathology
## Proteomics & Peptides                                   Proteomics & Peptides
## Soil Sciences                                                   Soil Sciences
## Sustainable Development                               Sustainable Development
## Sustainable Energy                                         Sustainable Energy
## Virology                                                             Virology
## Wood Science & Technology                           Wood Science & Technology
## Zoology                                                               Zoology
##                                        Between Degree Closeness     Eigen
## Agronomy Crop Science                       27      3 0.0021289 2.034e-03
## Animal Behavior                             27      5 0.0021497 0.000e+00
## Animal Husbandry                            19      2 0.0021439 1.008e-03
## Atmospheric Sciences                         4      3 0.0021438 1.086e-03
## Biochemistry                                 0      2 0.0007734 7.085e-01
## Biodiversity & Conservation Biology          2      4 0.0021450 0.000e+00
## Bioinformatics                               0      0 0.0007112 3.393e-04
## Biophysics                                   0      1 0.0007309 2.216e-04
## Biotechnology                               36      2 0.0021514 8.160e-04
## Birds                                       22      3 0.0021504 1.955e-04
## Botany                                      17      2 0.0021263 5.168e-04
## Cell Biology                                 1      3 0.0007736 9.338e-01
## Developmental Biology & Embryology           1      2 0.0007736 2.032e-01
## Ecology                                    134      8 0.0021555 0.000e+00
## Environmental & Geological Engineering       0      1 0.0021513 2.784e-04
## Environmental Sciences                     166      6 0.0021582 1.146e-03
## Evolutionary Biology                         0      3 0.0021468 0.000e+00
## Food Science & Technology                   22      2 0.0021449 8.486e-04
## Forests & Forestry                           0      2 0.0021502 0.000e+00
## Geochemistry & Mineralogy                    0      1 0.0021069 4.423e-05
## Geology                                     47      3 0.0021313 0.000e+00
## Hydrology                                    4      2 0.0021459 6.002e-04
## Insects & Arthropods                        36      2 0.0021513 4.648e-04
## Marine Sciences Fisheries                   94      3 0.0021559 0.000e+00
## Microbiology                                26      2 0.0021462 7.396e-04
## Molecular Biology                            0      3 0.0007736 1.000e+00
## Mycology                                     0      0 0.0007112 3.393e-04
## Oceanography                                82      4 0.0021515 1.646e-04
## Paleontology                                 0      1 0.0021064 4.589e-05
## Pest Control & Pesticides                   20      2 0.0021284 1.072e-03
## Plant Pathology                             14      3 0.0021253 1.459e-03
## Proteomics & Peptides                        0      1 0.0007309 2.216e-04
## Soil Sciences                               59      3 0.0021540 1.662e-03
## Sustainable Development                      0      0 0.0007112 3.393e-04
## Sustainable Energy                           0      0 0.0007112 3.393e-04
## Virology                                     0      0 0.0007112 3.393e-04
## Wood Science & Technology                    0      0 0.0007112 3.393e-04
## Zoology                                      0      4 0.0021460 0.000e+00
## 
## $`2003`
##                                                                         Class
## Agronomy Crop Science                                   Agronomy Crop Science
## Animal Behavior                                               Animal Behavior
## Animal Husbandry                                             Animal Husbandry
## Atmospheric Sciences                                     Atmospheric Sciences
## Biochemistry                                                     Biochemistry
## Biodiversity & Conservation Biology       Biodiversity & Conservation Biology
## Bioinformatics                                                 Bioinformatics
## Biophysics                                                         Biophysics
## Biotechnology                                                   Biotechnology
## Birds                                                                   Birds
## Botany                                                                 Botany
## Cell Biology                                                     Cell Biology
## Developmental Biology & Embryology         Developmental Biology & Embryology
## Ecology                                                               Ecology
## Environmental & Geological Engineering Environmental & Geological Engineering
## Environmental Sciences                                 Environmental Sciences
## Evolutionary Biology                                     Evolutionary Biology
## Food Science & Technology                           Food Science & Technology
## Forests & Forestry                                         Forests & Forestry
## Geochemistry & Mineralogy                           Geochemistry & Mineralogy
## Geology                                                               Geology
## Hydrology                                                           Hydrology
## Insects & Arthropods                                     Insects & Arthropods
## Marine Sciences Fisheries                           Marine Sciences Fisheries
## Microbiology                                                     Microbiology
## Molecular Biology                                           Molecular Biology
## Mycology                                                             Mycology
## Oceanography                                                     Oceanography
## Paleontology                                                     Paleontology
## Pest Control & Pesticides                           Pest Control & Pesticides
## Plant Pathology                                               Plant Pathology
## Proteomics & Peptides                                   Proteomics & Peptides
## Soil Sciences                                                   Soil Sciences
## Sustainable Development                               Sustainable Development
## Sustainable Energy                                         Sustainable Energy
## Virology                                                             Virology
## Wood Science & Technology                           Wood Science & Technology
## Zoology                                                               Zoology
##                                        Between Degree Closeness     Eigen
## Agronomy Crop Science                        0      3 0.0025617 0.0008228
## Animal Behavior                             55      5 0.0025814 0.0001545
## Animal Husbandry                            36      2 0.0025744 0.0010058
## Atmospheric Sciences                         4      3 0.0025619 0.0006393
## Biochemistry                                 6      3 0.0008214 0.7637118
## Biodiversity & Conservation Biology         93      5 0.0025839 0.0000000
## Bioinformatics                               0      0 0.0007112 0.0001073
## Biophysics                                   0      1 0.0008212 0.0033143
## Biotechnology                               33      2 0.0025730 0.0010267
## Birds                                        2      3 0.0025784 0.0001204
## Botany                                     104      3 0.0025773 0.0006245
## Cell Biology                                 6      3 0.0008214 0.9126029
## Developmental Biology & Embryology           4      2 0.0008213 0.1686625
## Ecology                                     49      7 0.0025831 0.0000000
## Environmental & Geological Engineering       0      1 0.0025727 0.0003717
## Environmental Sciences                     139      6 0.0025830 0.0004583
## Evolutionary Biology                       119      5 0.0025832 0.0000000
## Food Science & Technology                   25      2 0.0025677 0.0011638
## Forests & Forestry                         123      3 0.0025855 0.0000000
## Geochemistry & Mineralogy                    0      1 0.0025041 0.0001895
## Geology                                     51      3 0.0025499 0.0001057
## Hydrology                                   15      2 0.0025681 0.0005428
## Insects & Arthropods                         0      1 0.0024962 0.0003796
## Marine Sciences Fisheries                    0      3 0.0025691 0.0000000
## Microbiology                                37      3 0.0025672 0.0014992
## Molecular Biology                            0      3 0.0008211 1.0000000
## Mycology                                     0      1 0.0025578 0.0004237
## Oceanography                                79      4 0.0025729 0.0000000
## Paleontology                                 0      1 0.0025110 0.0002272
## Pest Control & Pesticides                   26      2 0.0025444 0.0006727
## Plant Pathology                             74      4 0.0025668 0.0009060
## Proteomics & Peptides                        4      2 0.0008214 0.0401692
## Soil Sciences                              136      4 0.0025847 0.0005743
## Sustainable Development                      0      0 0.0007112 0.0001073
## Sustainable Energy                           0      0 0.0007112 0.0001073
## Virology                                     0      1 0.0025569 0.0005894
## Wood Science & Technology                    0      0 0.0007112 0.0001073
## Zoology                                      0      4 0.0025737 0.0000000
## 
## $`2005`
##                                                                         Class
## Agronomy Crop Science                                   Agronomy Crop Science
## Animal Behavior                                               Animal Behavior
## Animal Husbandry                                             Animal Husbandry
## Atmospheric Sciences                                     Atmospheric Sciences
## Biochemistry                                                     Biochemistry
## Biodiversity & Conservation Biology       Biodiversity & Conservation Biology
## Bioinformatics                                                 Bioinformatics
## Biophysics                                                         Biophysics
## Biotechnology                                                   Biotechnology
## Birds                                                                   Birds
## Botany                                                                 Botany
## Cell Biology                                                     Cell Biology
## Developmental Biology & Embryology         Developmental Biology & Embryology
## Ecology                                                               Ecology
## Environmental & Geological Engineering Environmental & Geological Engineering
## Environmental Sciences                                 Environmental Sciences
## Evolutionary Biology                                     Evolutionary Biology
## Food Science & Technology                           Food Science & Technology
## Forests & Forestry                                         Forests & Forestry
## Geochemistry & Mineralogy                           Geochemistry & Mineralogy
## Geology                                                               Geology
## Hydrology                                                           Hydrology
## Insects & Arthropods                                     Insects & Arthropods
## Marine Sciences Fisheries                           Marine Sciences Fisheries
## Microbiology                                                     Microbiology
## Molecular Biology                                           Molecular Biology
## Mycology                                                             Mycology
## Oceanography                                                     Oceanography
## Paleontology                                                     Paleontology
## Pest Control & Pesticides                           Pest Control & Pesticides
## Plant Pathology                                               Plant Pathology
## Proteomics & Peptides                                   Proteomics & Peptides
## Soil Sciences                                                   Soil Sciences
## Sustainable Development                               Sustainable Development
## Sustainable Energy                                         Sustainable Energy
## Virology                                                             Virology
## Wood Science & Technology                           Wood Science & Technology
## Zoology                                                               Zoology
##                                        Between Degree Closeness     Eigen
## Agronomy Crop Science                       40      3 0.0025272 1.089e-03
## Animal Behavior                             49      5 0.0025643 0.000e+00
## Animal Husbandry                            31      2 0.0025551 3.452e-04
## Atmospheric Sciences                         4      3 0.0025545 7.333e-04
## Biochemistry                                 0      2 0.0007733 7.519e-01
## Biodiversity & Conservation Biology         12      5 0.0025594 0.000e+00
## Bioinformatics                               0      0 0.0007112 1.345e-04
## Biophysics                                   0      1 0.0007310 1.069e-05
## Biotechnology                               40      2 0.0025631 7.572e-04
## Birds                                        0      3 0.0025618 0.000e+00
## Botany                                      29      2 0.0025194 2.195e-04
## Cell Biology                                 2      3 0.0007737 9.066e-01
## Developmental Biology & Embryology           2      2 0.0007737 1.779e-01
## Ecology                                    112      7 0.0025702 0.000e+00
## Environmental & Geological Engineering       0      1 0.0025627 2.207e-04
## Environmental Sciences                     200      7 0.0025761 1.111e-03
## Evolutionary Biology                         5      4 0.0025591 0.000e+00
## Food Science & Technology                   23      2 0.0025551 3.529e-04
## Forests & Forestry                          14      2 0.0025622 0.000e+00
## Geochemistry & Mineralogy                    0      1 0.0025117 3.229e-04
## Geology                                     51      3 0.0025527 4.954e-04
## Hydrology                                    5      2 0.0025560 3.544e-04
## Insects & Arthropods                        30      2 0.0025558 3.691e-04
## Marine Sciences Fisheries                  103      3 0.0025709 0.000e+00
## Microbiology                                42      3 0.0025570 5.703e-04
## Molecular Biology                            0      3 0.0007736 1.000e+00
## Mycology                                     0      0 0.0007112 1.345e-04
## Oceanography                                89      4 0.0025676 7.220e-04
## Paleontology                                 0      1 0.0025064 3.744e-04
## Pest Control & Pesticides                   21      2 0.0025150 4.695e-04
## Plant Pathology                             24      3 0.0025166 7.327e-04
## Proteomics & Peptides                        0      1 0.0007310 1.068e-05
## Soil Sciences                               75      3 0.0025701 9.237e-04
## Sustainable Development                      0      1 0.0025640 1.921e-04
## Sustainable Energy                           0      0 0.0007112 1.345e-04
## Virology                                     0      1 0.0025468 1.221e-04
## Wood Science & Technology                    0      0 0.0007112 1.345e-04
## Zoology                                     48      5 0.0025633 0.000e+00
yearstats<-melt(yearstats)
ggplot(yearstats,aes(x=L1,y=value,col=Class)) + geom_point() +geom_line(aes(group=Class)) + facet_wrap(~variable,scales="free",ncol=1)

plot of chunk unnamed-chunk-15

Connectance through time

yeardat<-lapply(yearcompare,function(x){
  siteXspp<-droplevels(as.data.frame.array(table(x$Author,x$Class)))
  
  #drop empty rows and colums
  siteXspp<-siteXspp[rownames(siteXspp) %in% names(which(apply(siteXspp,1,sum) > 0)),colnames(siteXspp) %in% names(which(apply(siteXspp,2,sum) > 0))]
  
  #Compare disciplines
  topics<-1-as.matrix(vegdist(t(siteXspp),"horn"))
  diag(topics)<-NA
  topics[upper.tri(topics)]<-NA
  return(topics)
})

yeardat<-melt(yeardat)
colnames(yeardat)<-c("To","From","Niche.Overlap","Year")

Remove extremely weak connections

#remove very weak connections
#work from an example data
#make into characters
yeardat$To<-as.character(yeardat$To)
yeardat$From<-as.character(yeardat$From)

#remove weak connections
exdat<-group_by(yeardat,To,From) %>% filter(Niche.Overlap>0.05 & !is.na(Niche.Overlap))

exdat$Combo<-paste(exdat$To,exdat$From,sep="-")

#plot
ggplot(exdat,aes(x=Year,y=Niche.Overlap,col=From)) + geom_point() + geom_line(aes(group=Combo)) + facet_wrap(~To,scales="free",ncol=3) + theme_bw() + geom_text(data=exdat[exdat$Year==m[length(m)/2],],aes(label=From),size=4) 

plot of chunk unnamed-chunk-17

ggsave("Figures/LinkTime.svg",dpi=300)
ggsave("Figures/LinkTime.jpeg",height=10,width=14,dpi=300)

trend estimation

Its not clear to me if i need to use a time-series model. The connection at time A should be independent of connection at time A+1. While they may be related due to the trend - there is no mechanistic connection among publications between years. Its not like a population, where the number of available producers directly influences the offspring in the next year. For the moment, i’m just using a linear model with year as a continious variable. This probably needs to change.

first pass just fit a linear line and determine if its positive or negative

exdat$Combo<-paste(exdat$To,exdat$From,sep="-")

#year is a number value for the moment
exdat$Year<-as.numeric(exdat$Year)-1995

sdat<-split(exdat,exdat$Combo)

#get rid of combinations with less than ten years of points
sdat<-sdat[lapply(sdat,nrow) > 2]

#
# Break up d by state, then fit the specified model to each piece and
# return a list
tmod<-rbind_all(lapply(sdat,function(df){
  tdat<-tidy(lm(Niche.Overlap ~ Year, data = df))
  tdat<-tdat[tdat$term =="Year",]
  tdat$Combo<-unique(df$Combo)
  return(tdat)
}))

#extract names into columns
tmod$To<-str_match(tmod$Combo,"(.*)-")[,2]
tmod$From<-str_match(tmod$Combo,"-(.*)")[,2]

Visualize effect over time

Positive values are significantly increasing connections Negative values are significantly decreasing connections

tmod$svalue<-scale(tmod$estimate)

ggplot(tmod[tmod$p.value < 0.05 & tmod$term=="Year",],aes(x=To,y=From,fill=estimate)) + geom_tile() + theme_bw() + scale_fill_gradientn(colours=c("blue","gray","red"),limits=c(-max(tmod$estimate),max(tmod$estimate))) + theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1)) + labs(x="",y="")

plot of chunk unnamed-chunk-19

ggsave("InteractionsTime.svg",dpi=300)
ggsave("InteractionsTime.jpeg",height=7,width=7) 

D3 Networks

D3 Visualizations

The wonderful D3 package connections!

MisLinks<-melt(topics)

#remove very weak connections
MisLinks<-MisLinks[MisLinks$value > 0.05,]
MisLinks$value<-MisLinks$value*10
colnames(MisLinks)<-c("To","From","value")
MisLinks$To<-as.character(MisLinks$To)
MisLinks$From<-as.character(MisLinks$From)

MisNodes<-data.frame(name=as.factor(sort(as.character(unique(c(MisLinks$To,MisLinks$From))))))
#Add groups
MisNodes$group<-as.integer(1)

MisLinks$source<-as.integer(sapply(MisLinks$To,function(x) which(x ==MisNodes$name)))-1
MisLinks$target<-as.integer(sapply(MisLinks$From,function(x) which(x ==MisNodes$name)))-1

#Order by source
MisLinks<-MisLinks[order(MisLinks$source),]

simpleNetwork(MisLinks,fontSize = 15)

simpleNetwork(MisLinks,height=500,width=700)  %>% saveNetwork(file = 'Net1.html',selfcontained=F)

Force based network

d3Network::d3ForceNetwork(Links = MisLinks, Nodes = MisNodes, Source = "source",Target = "target", Value = "value", NodeID = "name", Group = "group", opacity = 0.9,file="Net3.html") 
save.image("Analysis.RData")